#include "Randomizer.hh"
#include "Map.hh"
#include <iostream>
#include "Bug.h"
#include <vector>

Bug bug("test");
#ifdef DEBUG
int DEBUGTracker::depth = 1;
#endif
  
double identity(double x)
{
  return x;
}

CacheMap torus;
Map<double> expected_reward;
unsigned count;
uLocation target;
uLocation target2;
uLocation danger;
double alpha = 0.001;
const double epsilon = 0.01;
double lambda = 0.001;

double walk(uLocation start, unsigned n, unsigned t, unsigned T)
{
  std::vector<uLocation> path;
  double reward = 0;
  for(unsigned i=0;i!=n;++i)
    {
      path.push_back(start);
      /*
      if(start == target) reward += 1.0/pow(n+1,0.5);
      if(start == target2) reward += 1.0/pow(n+1,0.5);
      if(start == danger) reward -= 1.0/pow(n+1,0.5);
      */
      if(start == target) reward += 1.0/(n+1);
      if(start == target2) reward += 1.0/(n+1);
      if(start == danger) reward -= 1.0/(n+1);


      
      unsigned direction = rand()%4;
      if(rand()%2==0)
	{
	  double best_reward = -1e8;
	  
	  for(unsigned d=0;d!=5;++d)
	    {
	      uLocation nLoc = torus.getLocation(start,d);
	      if(expected_reward[nLoc] > best_reward or expected_reward[nLoc] == best_reward and rand()%2==0) 
		{
		  best_reward = expected_reward[nLoc];
		  direction = d;
		}
	    }
	}

      start = torus.getLocation(start,direction);
    }

 
  for(unsigned i=0; i!=path.size();++i)
    expected_reward[path[i]] = expected_reward[path[i]] + alpha*(reward-expected_reward[path[i]]);
  //alpha = 0.999*alpha;

  return reward;
}


  
int main(int argc, char** argv)
{
  srand(time(0));
  bug.init("test");


  unsigned step = 100;
  unsigned iter = 10000;
  if(argc>1) step = atoi(argv[1]);
  if(argc>2) iter = atoi(argv[2]);

  torus.init(100,100,1,1,1);
  expected_reward.init();
  count = 1;
  uLocation center = torus.to_uLocation(0,0);
  target = torus.to_uLocation(30,30);
  target2 = torus.to_uLocation(30,60);
  danger = torus.to_uLocation(30,50);


  for(uLocation loc=0;loc!=torus.size; ++loc)
    expected_reward[loc]=0.001;

  std::cerr << "Iterations: " << iter << ", Depth: " << step 
	    << std::endl;

  
  for(unsigned t=0;t!=iter;t++)
    {
      double cumr = walk(center,step,t,iter);
      if(fabs(cumr)>0)
	std::cerr << t << ' ' << cumr
		  << std::endl;
    }
  std::cerr << std::endl;

  for(unsigned row=0;row!=torus.rows;++row)
    {
      std::cout << expected_reward.at(row,0)/count;
      for(unsigned col=1;col<torus.cols;++col)
	{
	  std::cout << ' '
		    << expected_reward.at(row,col);
	}
      std::cout << std::endl;
    }
  std::cout << std::endl;
  return 0;
}
